/**
 * Copyright (C), 2002-2017, 上海巧房信息技术有限公司
 * FileName: ElementRuleServiceImpl
 * Author:   yi.li
 * Date:     2018/6/29 17:17
 * Description: 采集规则服务实现
 * History: //修改记录
 * <author>      <time>      <version>    <desc>
 * 修改人姓名             修改时间            版本号                  描述
 */
package com.qiaofang.collectmanage.service.impl;




import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.qiaofang.collectmanage.analysis.HtmlAnalysisConsts;
import com.qiaofang.collectmanage.analysis.HtmlAnalysisUtils;
import com.qiaofang.collectmanage.analysis.ReturnElement;
import com.qiaofang.collectmanage.constants.CollectRuleFieldEnum;
import com.qiaofang.collectmanage.constants.PersonalHouseRentEnum;
import com.qiaofang.collectmanage.constants.PersonalHouseSellEnum;
import com.qiaofang.collectmanage.dao.CollectRuleDao;
import com.qiaofang.collectmanage.dao.ElementRuleDao;
import com.qiaofang.collectmanage.dao.domain.CollectRule;
import com.qiaofang.collectmanage.dao.domain.ElementRule;
import com.qiaofang.collectmanage.service.ElementRuleService;
import com.qiaofang.collectmanage.service.common.utils.HttpClientUtils;
import com.qiaofang.collectmanage.service.util.ElementRuleServiceConvert;
import com.qiaofang.collectmanage.stub.bean.*;
import com.qiaofang.collectmanage.stub.constants.*;
import com.qiaofang.common.exception.BusinessException;
import com.qiaofang.common.model.page.PageDTO;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

@Service
public class ElementRuleServiceImpl implements ElementRuleService {
	private static final Logger LOGGER = LoggerFactory.getLogger(ElementRuleServiceImpl.class);

	@Autowired
	ElementRuleDao elementRuleDao;

	@Autowired
	CollectRuleDao collectRuleDao;

	@Autowired
	private RestTemplate restTemplate;

	@Bean
	public RestTemplate restTemplate(){
		RestTemplate restTemplate = new RestTemplate();
		return restTemplate;
	}

	@Override
	public ElementRuleDTO getElementRuleById(Integer id){
		ElementRule elementRule = elementRuleDao.getElementRuleById(id);
		return ElementRuleServiceConvert.convertDomain2DTO(elementRule);
	}

    @Override
    public List<ElementRuleDTO> getElementRuleByUUid(String collectRuleId){
        return elementRuleDao.getElementRuleByUUid(collectRuleId).stream().map(ElementRuleServiceConvert::convertDomain2DTO).collect(Collectors.toList());

    }


	@Override
	public void saveElementRule(SaveElementRuleDTO saveElementRuleDTO) {
	    if(!PersonalHouseRentEnum.valueExist(saveElementRuleDTO.getFieldName())&&!PersonalHouseSellEnum.valueExist(saveElementRuleDTO.getFieldName())){
			throw new BusinessException(ElementRuleResponseCode.ADD_INVALID_FIELD.getResponseCode(),
					ElementRuleResponseCode.ADD_INVALID_FIELD.getResponseMessage());
		}
		//查询元素采集规则是否有过
		ElementRuleDTO elementRule = new ElementRuleDTO();
        elementRule.setCollectRuleUuid(saveElementRuleDTO.getCollectRuleUuid());
        elementRule.setFieldName(saveElementRuleDTO.getFieldName());
		ElementRuleDTO elementRuleDTO = getElementRule(elementRule);
		if (elementRuleDTO != null) {
			if(DeletedEnum.NO.getValue().equals(elementRuleDTO.getDeleted())) {
				throw new BusinessException(ElementRuleResponseCode.ADD_ELEMENTRULE_DUPLICATE.getResponseCode(),
						ElementRuleResponseCode.ADD_ELEMENTRULE_DUPLICATE.getResponseMessage());
			}else{
				UpdateElementRuleDTO updateElementRuleDTO =ElementRuleServiceConvert.convertSaveDTO2UpdateDTO(saveElementRuleDTO);
				updateElementRuleDTO.setId(elementRuleDTO.getId());
				updateElementRuleDTO.setCollectRuleUuid(elementRuleDTO.getCollectRuleUuid());
				updateElementRuleDTO.setDeleted(DeletedEnum.NO.getValue());
				elementRuleDao.updateElementRule(ElementRuleServiceConvert.convertUpdateDTO2Domain(updateElementRuleDTO));
				return;
			}
		}
		//添加元素采集规则
		elementRuleDao.saveElementRule(ElementRuleServiceConvert.convertSaveDTO2Domain(saveElementRuleDTO));
	}

	@Override
	public ElementRuleDTO getElementRule(GetElementRuleDTO dto) {
		ElementRule domain = elementRuleDao.getElementRule(ElementRuleServiceConvert.convertGetDTO2Domain(dto));
		return ElementRuleServiceConvert.convertDomain2DTO(domain);
	}

	public ElementRuleDTO getElementRule(ElementRuleDTO dto) {
		ElementRule domain = elementRuleDao.getElementRule(ElementRuleServiceConvert.convertDTO2Domain(dto));
		return ElementRuleServiceConvert.convertDomain2DTO(domain);
	}


	@Override
	public int updateElementRule(UpdateElementRuleDTO updateElementRuleDTO) {
		if(!PersonalHouseRentEnum.valueExist(updateElementRuleDTO.getFieldName())&&!PersonalHouseSellEnum.valueExist(updateElementRuleDTO.getFieldName())){
			throw new BusinessException(ElementRuleResponseCode.ADD_INVALID_FIELD.getResponseCode(),
					ElementRuleResponseCode.ADD_INVALID_FIELD.getResponseMessage());
		}
        return elementRuleDao.updateElementRule(ElementRuleServiceConvert.convertUpdateDTO2Domain(updateElementRuleDTO));
	}

	@Override
	public List<ElementRuleDTO> listElementRule(ListElementRuleDTO elementRuleDTO) {
		return elementRuleDao.listElementRule(ElementRuleServiceConvert.convertListDTO2Domain(elementRuleDTO)).stream().map(ElementRuleServiceConvert::convertDomain2DTO).collect(Collectors.toList());
	}

	@Override
	public int removeElementRule(RemoveElementRuleDTO removeElementRuleDTO) {
		return elementRuleDao.removeElementRule(removeElementRuleDTO.getId());
	}

	@Override
	public List<ElementRuleDTO> searchElementRulePage(SearchElementRulePostDTO searchElementRulePostDTO, PageDTO page) {
		if(page==null){ page  = new PageDTO();}
		List<ElementRule> list = elementRuleDao.searchElementRulePage(searchElementRulePostDTO, page);
		if(list!=null) {
			List<ElementRuleDTO> listDTO = list.stream().map(ElementRuleServiceConvert::convertDomain2DTO).collect(Collectors.toList());
			return listDTO;
		}
		return null;
	}

	@Override
	public Long countSearchElementRulePage(SearchElementRulePostDTO dto) {
		return elementRuleDao.countSearchElementRulePage(dto);
	}

	@Override
	public List<TestFieldsByIdDTO> testFieldsById(List<Integer> ids) {
		Long functionStart = System.currentTimeMillis();
		List<TestFieldsByIdDTO> testFieldsByIdDTOs = new ArrayList<>();
		try {
			Integer idFrist = 0;
			if (ids.size() > 0) {
				idFrist = ids.get(0);
			}
			ElementRule elementRule = elementRuleDao.getElementRuleById(idFrist);
			CollectRule collectRule = collectRuleDao.getCollectRuleByUuid(elementRule.getCollectRuleUuid());

			JSONObject jsonObject = new JSONObject();
			jsonObject.put(CollectRuleFieldEnum.CITY.getValue(),collectRule.getCity());
			jsonObject.put(CollectRuleFieldEnum.BELONGER.getValue(),collectRule.getBelonger());
			jsonObject.put(CollectRuleFieldEnum.SOURCE.getValue(),collectRule.getSource());
			jsonObject.put(CollectRuleFieldEnum.COLLECT_HOUSE_TYPE.getValue(),collectRule.getCollectHouseType());
			jsonObject.put(CollectRuleFieldEnum.COLLECT_TYPE.getValue(),collectRule.getCollectType());
			jsonObject.put(CollectRuleFieldEnum.WEBSITE.getValue(),collectRule.getWebsite());

			//获取uuid相同的所有的页面元素规则
			List<ElementRule> allElementRule = elementRuleDao.getElementRuleByUUid(elementRule.getCollectRuleUuid());

			String url = collectRule.getWebsite().replace("{}", "1");
			LOGGER.info("列表页面链接{}", url);
			String pageText = HttpClientUtils.getPageTextByProxy(url, collectRule.getListWebCode());
			int count = 0;
			Long start = System.currentTimeMillis();
			while (StringUtils.isBlank(pageText.toString())
					|| pageText.toString().contains(HtmlAnalysisConsts.WB_ERROR_TEXT)
					|| pageText.toString().contains(HtmlAnalysisConsts.FANG_ERROR_TEXT)
					|| pageText.toString().contains(HtmlAnalysisConsts.GJ_ERROR_TEXT)
					|| pageText.toString().contains(HtmlAnalysisConsts.AJK_ERROR_TEXT)) {
				pageText = HttpClientUtils.getPageTextByProxy(url, collectRule.getListWebCode());
				count++;
				if(count==5) {
					throw new BusinessException(CollectRuleResponseCode.COLLECT_TIMEOUT.getResponseCode(),CollectRuleResponseCode.COLLECT_TIMEOUT.getResponseMessage());
				}
			}
			LOGGER.info("获取列表页面总耗时{}s循环次数{},页面链接{}", (System.currentTimeMillis() - start)/1000, count, url);
			Document pageDoc = Jsoup.parse(pageText);
			//获取房源列表规则 ,倒数第2深度为list
			int depthMax = elementRuleDao.maxDepth(elementRule);
			ElementRule paraElementRule = new ElementRule();
			paraElementRule.setDepth(depthMax - 2);
			paraElementRule.setCollectRuleUuid(elementRule.getCollectRuleUuid());
			List<ElementRule> list = elementRuleDao.listElementRule(paraElementRule);

			ElementRule singleElementRule = list.get(0);
			String[] rules = HtmlAnalysisConsts.PARAMETER_SEPARATOR.split(singleElementRule.getReturnElement());
			//房源列表元素
			Elements listHouse = ReturnElement.select(pageDoc, rules[1]);

			//测试list字段，则范围测试集合
			if (ids.size()>0 ) {
				for(Integer id:ids){
					ElementRule elementRuleFieldList = elementRuleDao.getElementRuleById(id);
					if("".equals(elementRuleFieldList.getFieldName())){
						TestFieldsByIdDTO testFieldsByIdDTO = new TestFieldsByIdDTO();
						testFieldsByIdDTO.setFieldName("list");
						StringBuilder sb = new StringBuilder();
						if(listHouse.size()>0) {
							sb.append(listHouse.get(0).html());
						}
						testFieldsByIdDTO.setAnalysisResult(sb.toString());
						testFieldsByIdDTOs.add(testFieldsByIdDTO);
					}
				}
			}

			if (listHouse.size() > 0) {
				Element pageFirstElement = listHouse.get(0);
				//获取详情页面元素
				Document detailDoc = null;
				//获取title的页面规则
				List<ElementRule> titleElementRules = allElementRule.stream().filter(elementRule1 -> "titleLink".equals(elementRule1.getFieldName())).collect(Collectors.toList());

				if (titleElementRules.size() > 0) {
					jsonObject = HtmlAnalysisUtils.analysisRule(pageFirstElement, titleElementRules.stream().map(ElementRuleServiceConvert::convertDomain2Domain).collect(Collectors.toList()), jsonObject);
				}
				String titleLink = (String) jsonObject.get(HtmlAnalysisConsts.TITLE_LINK);
				if (!titleLink.startsWith("http")){ titleLink = "http:" + titleLink;}
				LOGGER.info("详情链接{}", titleLink);
				Long start2 = System.currentTimeMillis();
				if (StringUtils.isNotBlank(titleLink)) {
					String detailHtml = HttpClientUtils.getPageTextByProxy(titleLink, collectRule.getDetailWebCode());
					while (StringUtils.isBlank(String.valueOf(detailHtml))
							//|| detailHtml.toString().contains(HtmlAnalysisConsts.WB_ERROR_TEXT)
							|| detailHtml.toString().contains(HtmlAnalysisConsts.FANG_ERROR_TEXT)
							|| detailHtml.toString().contains(HtmlAnalysisConsts.GJ_ERROR_TEXT)
							|| detailHtml.toString().contains(HtmlAnalysisConsts.AJK_ERROR_TEXT)) {
						detailHtml = HttpClientUtils.getPageTextByProxy(titleLink, collectRule.getDetailWebCode());
					}
					detailDoc = Jsoup.parse(String.valueOf(detailHtml));
				}
				LOGGER.info("获取详情页面总耗时{}s循环次数{},页面链接{}", ( System.currentTimeMillis()- start2 )/1000, count, url);

				for (Integer id : ids) {
					TestFieldsByIdDTO testFieldsByIdDTO = new TestFieldsByIdDTO();
					ElementRule elementRuleTest = elementRuleDao.getElementRuleById(id);
					if(elementRuleTest!=null&&!"".equals(elementRuleTest.getFieldName())){
						int depthT = elementRuleTest.getDepth();
						int diff = depthMax-depthT;
						if (diff == 0 && detailDoc != null) {
							String returnFieldValue = HtmlAnalysisUtils.analysisRule(detailDoc, ElementRuleServiceConvert.convertDomain2Domain(elementRuleTest),jsonObject).toString();
                            returnFieldValue = StringUtils.isBlank(returnFieldValue) ? null : returnFieldValue.toString().trim();
							testFieldsByIdDTO.setAnalysisResult(returnFieldValue);
						} else if (diff > 0  && pageFirstElement != null) {
							String returnFieldValue =HtmlAnalysisUtils.analysisRule(pageFirstElement, ElementRuleServiceConvert.convertDomain2Domain(elementRuleTest),jsonObject).toString();
							returnFieldValue = StringUtils.isBlank(returnFieldValue) ? null : returnFieldValue.toString().trim();
							testFieldsByIdDTO.setAnalysisResult(returnFieldValue);
						}
						testFieldsByIdDTO.setFieldName(elementRuleTest.getFieldName());
						testFieldsByIdDTOs.add(testFieldsByIdDTO);
					}
				}
				LOGGER.info(">>>>>>检测结束总耗时{}s", (System.currentTimeMillis() - functionStart)/1000);
			}
		}catch (Exception e){
            throw new BusinessException(CollectRuleResponseCode.COLLECT_FAIL.getResponseCode(),e.getMessage());
        }
		return testFieldsByIdDTOs;
	}


	@Override
	public List<TestFieldsByIdDTO> testToolFieldsById(List<Integer> ids,String srcTitleLink) {
		Long functionStart = System.currentTimeMillis();
		List<TestFieldsByIdDTO> testFieldsByIdDTOs = new ArrayList<>();
		try {
			Integer idFrist = 0;
			if (ids.size() > 0) {
				idFrist = ids.get(0);
			}
			ElementRule elementRule = elementRuleDao.getElementRuleById(idFrist);
			CollectRule collectRule = collectRuleDao.getCollectRuleByUuid(elementRule.getCollectRuleUuid());

			JSONObject jsonObject = new JSONObject();
			jsonObject.put(CollectRuleFieldEnum.CITY.getValue(),collectRule.getCity());
			jsonObject.put(CollectRuleFieldEnum.BELONGER.getValue(),collectRule.getBelonger());
			jsonObject.put(CollectRuleFieldEnum.SOURCE.getValue(),collectRule.getSource());
			jsonObject.put(CollectRuleFieldEnum.COLLECT_HOUSE_TYPE.getValue(),collectRule.getCollectHouseType());
			jsonObject.put(CollectRuleFieldEnum.COLLECT_TYPE.getValue(),collectRule.getCollectType());
			jsonObject.put(CollectRuleFieldEnum.WEBSITE.getValue(),collectRule.getWebsite());

			//获取uuid相同的所有的页面元素规则
			List<ElementRule> allElementRule = elementRuleDao.getElementRuleByUUid(elementRule.getCollectRuleUuid());

			String url = collectRule.getWebsite().replace("{}", "1");
			LOGGER.info("列表页面链接{}", url);
			String pageText = HttpClientUtils.getPageTextByProxy(url, collectRule.getListWebCode());
			int count = 0;
			Long start = System.currentTimeMillis();
			while (StringUtils.isBlank(pageText.toString())
					|| pageText.toString().contains(HtmlAnalysisConsts.WB_ERROR_TEXT)
					|| pageText.toString().contains(HtmlAnalysisConsts.FANG_ERROR_TEXT)
					|| pageText.toString().contains(HtmlAnalysisConsts.GJ_ERROR_TEXT)
					|| pageText.toString().contains(HtmlAnalysisConsts.AJK_ERROR_TEXT)) {
				pageText = HttpClientUtils.getPageTextByProxy(url, collectRule.getListWebCode());
				count++;
				if(count==5) {
					throw new BusinessException(CollectRuleResponseCode.COLLECT_TIMEOUT.getResponseCode(),CollectRuleResponseCode.COLLECT_TIMEOUT.getResponseMessage());
				}
			}
			LOGGER.info("获取列表页面总耗时{}s循环次数{},页面链接{}", (System.currentTimeMillis() - start)/1000, count, url);
			Document pageDoc = Jsoup.parse(pageText);
			//获取房源列表规则 ,倒数第2深度为list
			int depthMax = elementRuleDao.maxDepth(elementRule);
			ElementRule paraElementRule = new ElementRule();
			paraElementRule.setDepth(depthMax - 2);
			paraElementRule.setCollectRuleUuid(elementRule.getCollectRuleUuid());
			List<ElementRule> list = elementRuleDao.listElementRule(paraElementRule);

			ElementRule singleElementRule = list.get(0);
			String[] rules = HtmlAnalysisConsts.PARAMETER_SEPARATOR.split(singleElementRule.getReturnElement());
			//房源列表元素
			Elements listHouse = ReturnElement.select(pageDoc, rules[1]);

			//测试list字段，则范围测试集合
			if (ids.size()>0 ) {
				for(Integer id:ids){
					ElementRule elementRuleFieldList = elementRuleDao.getElementRuleById(id);
					if("".equals(elementRuleFieldList.getFieldName())){
						TestFieldsByIdDTO testFieldsByIdDTO = new TestFieldsByIdDTO();
						testFieldsByIdDTO.setFieldName("list");
						StringBuilder sb = new StringBuilder();
						if(listHouse.size()>0) {
							sb.append(listHouse.get(0).html());
						}
						testFieldsByIdDTO.setAnalysisResult(sb.toString());
						testFieldsByIdDTOs.add(testFieldsByIdDTO);
					}
				}
			}

			if (listHouse.size() > 0) {
				Element pageFirstElement = listHouse.get(0);
				//获取详情页面元素
				Document detailDoc = null;
				//获取title的页面规则
				List<ElementRule> titleElementRules = allElementRule.stream().filter(elementRule1 -> "titleLink".equals(elementRule1.getFieldName())).collect(Collectors.toList());

				if (titleElementRules.size() > 0) {
					jsonObject = HtmlAnalysisUtils.analysisRule(pageFirstElement, titleElementRules.stream().map(ElementRuleServiceConvert::convertDomain2Domain).collect(Collectors.toList()), jsonObject);
				}
				String titleLink = (String) jsonObject.get(HtmlAnalysisConsts.TITLE_LINK);
				if (!titleLink.startsWith("http")){ titleLink = "http:" + titleLink;}
				titleLink = srcTitleLink;
				LOGGER.info("详情链接{}", titleLink);
				Long start2 = System.currentTimeMillis();
				if (StringUtils.isNotBlank(titleLink)) {
					String detailHtml = HttpClientUtils.getPageTextByProxy(titleLink, collectRule.getDetailWebCode());
					while (StringUtils.isBlank(String.valueOf(detailHtml))
							//|| detailHtml.toString().contains(HtmlAnalysisConsts.WB_ERROR_TEXT)
							|| detailHtml.toString().contains(HtmlAnalysisConsts.FANG_ERROR_TEXT)
							|| detailHtml.toString().contains(HtmlAnalysisConsts.GJ_ERROR_TEXT)
							|| detailHtml.toString().contains(HtmlAnalysisConsts.AJK_ERROR_TEXT)) {
						detailHtml = HttpClientUtils.getPageTextByProxy(titleLink, collectRule.getDetailWebCode());
					}
					detailDoc = Jsoup.parse(String.valueOf(detailHtml));
				}
				LOGGER.info("获取详情页面总耗时{}s循环次数{},页面链接{}", ( System.currentTimeMillis()- start2 )/1000, count, url);

				for (Integer id : ids) {
					TestFieldsByIdDTO testFieldsByIdDTO = new TestFieldsByIdDTO();
					ElementRule elementRuleTest = elementRuleDao.getElementRuleById(id);
					if(elementRuleTest!=null&&!"".equals(elementRuleTest.getFieldName())){
						int depthT = elementRuleTest.getDepth();
						int diff = depthMax-depthT;
						if (diff == 0 && detailDoc != null) {
							String returnFieldValue = HtmlAnalysisUtils.analysisRule(detailDoc, ElementRuleServiceConvert.convertDomain2Domain(elementRuleTest),jsonObject).toString();
							returnFieldValue = StringUtils.isBlank(returnFieldValue) ? null : returnFieldValue.toString().trim();
							testFieldsByIdDTO.setAnalysisResult(returnFieldValue);
						} else if (diff > 0  && pageFirstElement != null) {
							String returnFieldValue =HtmlAnalysisUtils.analysisRule(pageFirstElement, ElementRuleServiceConvert.convertDomain2Domain(elementRuleTest),jsonObject).toString();
							returnFieldValue = StringUtils.isBlank(returnFieldValue) ? null : returnFieldValue.toString().trim();
							testFieldsByIdDTO.setAnalysisResult(returnFieldValue);
						}
						testFieldsByIdDTO.setFieldName(elementRuleTest.getFieldName());
						testFieldsByIdDTOs.add(testFieldsByIdDTO);
					}
				}
				LOGGER.info(">>>>>>检测结束总耗时{}s", (System.currentTimeMillis() - functionStart)/1000);
			}
		}catch (Exception e){
			throw new BusinessException(CollectRuleResponseCode.COLLECT_FAIL.getResponseCode(),e.getMessage());
		}
		return testFieldsByIdDTOs;
	}

    @Override
    public List<String> getFields(String collectRuleUuid) {
        CollectRule collectRule =  collectRuleDao.getCollectRuleByUuid(collectRuleUuid);
        List<String> data = new ArrayList<>();
        if(collectRule!=null&&BelongerEnum.PERSONAL.getValue().equals(collectRule.getBelonger())){
            if(collectRule!=null&& CollectHouseTypeEnum.RESIDENTIAL.getValue().equals(collectRule.getCollectHouseType())){
                if(collectRule!=null&& CollectTypeEnum.RENT.getValue().equals(collectRule.getCollectType())){
                    data = PersonalHouseRentEnum.toValueList();
                }else if(collectRule!=null&& CollectTypeEnum.RENT.getValue().equals(collectRule.getCollectType())){
                    data = PersonalHouseSellEnum.toValueList();
                }
            }
        }
        List<ElementRuleDTO> elementRuleDTOS = getElementRuleByUUid(collectRuleUuid);
        for(ElementRuleDTO elementRuleDTO : elementRuleDTOS){
            String fieldName = elementRuleDTO.getFieldName();
            if(data.contains(fieldName)){
                data.remove(fieldName);
            }
        }
        return data;
    }

	@Override
	public List<String> getExistFields(String collectRuleUuid) {
		List<ElementRuleDTO> elementRuleDTOS = getElementRuleByUUid(collectRuleUuid);
		List<String> existData=elementRuleDTOS.stream()
				.filter((ElementRuleDTO elementRuleDTO)->!"".equals(elementRuleDTO.getFieldName()))
				.map(ElementRuleDTO::getFieldName).collect(Collectors.toList());
		return existData;
	}

}
